package org.huangrui.spark.scala.core.rdd.dep

import org.apache.spark.rdd.RDD
import org.apache.spark.storage.StorageLevel
import org.apache.spark.{SparkConf, SparkContext}

/**
 *
 * @Author hr
 * @Create 2024-10-19 11:30
 */
object Spark04_Dep_Persist {
  def main(args: Array[String]): Unit = {
    val sparConf = new SparkConf().setMaster("local[*]").setAppName("WordCount")
    val sc = new SparkContext(sparConf)

    val lines: RDD[String] = sc.makeRDD(Array("hello world", "hello atguigu", "atguigu", "hahah"))
    val words: RDD[String] = lines.flatMap(_.split(" "))
    val wordToOne = words.map(word => {
      println("***********************")
      (word, 1)
    })
    // cache默认持久化的操作，只能将数据保存到内存中，如果想要保存到磁盘文件，需要更改存储级别
     wordToOne.cache(); // wordToOne.persist(StorageLevel.MEMORY_ONLY)
    // 持久化操作必须在行动算子执行时完成的。
    wordToOne.persist(StorageLevel.DISK_ONLY)
    val wordToSum: RDD[(String, Int)] = wordToOne.reduceByKey(_ + _)
    val array: Array[(String, Int)] = wordToSum.collect()

    println("计算1完毕")
    array.foreach(println)
    println("#########################")
    wordToOne.groupByKey().collect()
    println("计算2完毕")

    sc.stop()

  }
}
